In [1]:
%reload_ext autotime
import pandas as pd
from bertopic import BERTopic
from bertopic.representation import KeyBERTInspired
import plotly.io as pio
pio.renderers.default = 'notebook'
pd.options.plotting.backend = "plotly"
df = pd.read_excel("givealittle_health.xlsx")
df.index = pd.to_datetime(df["start"])
df["text"] = df.title.str.cat(df[["pitch", "description", "use_of_funds"]].astype(str), sep=" ")
df
✔️ 12.1 s (2024-11-04T13:24:25/2024-11-04T13:24:37)
Out[1]:
| title | pitch | hero | uri | amountRaised | goal | timeLeft | action | actionUri | start | ... | location | n_donors | description | use_of_funds | whos_involved | moderated | n_updates | updates | Q&A | text | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| start | |||||||||||||||||||||
| 2024-11-03 | Young mother of 3 recently diagnosed with aggr... | Young mother of three fighting aggresive breas... | https://static.givealittle.co.nz/assets/hero/e... | https://givealittle.co.nz/cause/young-mother-o... | 3900.00 | 0.0 | 181 days to go | Donate | https://givealittle.co.nz/donate/cause/young-m... | 3 Nov 2024 | ... | Pirimai, Napier | 38.0 | Imagine facing one of life's biggest battles w... | Travel costs for treatment.Financial help with... | Created by_x000D_\nDebra Evans \n\n\n\n_x000D... | Page Moderated | NaN | NaN | NaN | Young mother of 3 recently diagnosed with aggr... |
| 2024-11-01 | Finlee Bremners Road to recovery | Helping our great friends navigate Finlee's lo... | https://static.givealittle.co.nz/assets/hero/2... | https://givealittle.co.nz/cause/finlee-bremner... | 4500.00 | 0.0 | 177 days to go | Donate | https://givealittle.co.nz/donate/cause/finlee-... | 1 Nov 2024 | ... | Alexandra, Otago | 44.0 | On the 2nd October 2024, Kayla, Michael and Ar... | All money received will go towards on going co... | Created by_x000D_\nKelsey McLean \n\n\n\n_x00... | Page Moderated | NaN | NaN | NaN | Finlee Bremners Road to recovery Helping our g... |
| 2024-10-31 | Paula Matafahi's Grade 4 Brain Cancer Fight - ... | This page has been set up to help Paula access... | https://static.givealittle.co.nz/assets/hero/b... | https://givealittle.co.nz/cause/paulas-stage-4... | 47522.15 | 55000.0 | 27 days to go | Donate | https://givealittle.co.nz/donate/cause/paulas-... | 31 Oct 2024 | ... | Manawatu / Whanganui | 628.0 | Paula is an extraordinary 18 year old young ma... | Providing the potentially life extending but u... | Created by_x000D_\nSheree Turia \n\n\n\n_x000... | Page Moderated | 2.0 | Nearly there!\n3 November 2024\nWe are just ab... | NaN | Paula Matafahi's Grade 4 Brain Cancer Fight - ... |
| 2024-10-31 | Our Sweet boy Jairo | Please help Jairo get the testing and screenin... | https://static.givealittle.co.nz/assets/hero/2... | https://givealittle.co.nz/cause/our-sweet-boy-... | 925.00 | 1000.0 | 19 days to go | Donate | https://givealittle.co.nz/donate/cause/our-swe... | 31 Oct 2024 | ... | Palmerston North, Manawatu-Wanganui | 9.0 | My 8 week old baby Jairo has had chronic colli... | It will be used for the consult and further te... | Created by, and paying to a verified bank acco... | Page Moderated | 1.0 | Follow up appointments\n2 November 2024\nJust ... | NaN | Our Sweet boy Jairo Please help Jairo get the ... |
| 2024-10-30 | Em’s MS Journey | Stemcell replacement for Em’s MS Journey. Clín... | https://static.givealittle.co.nz/assets/hero/6... | https://givealittle.co.nz/cause/ems-ms-journey | 9640.00 | 90000.0 | 178 days to go | Donate | https://givealittle.co.nz/donate/cause/ems-ms-... | 30 Oct 2024 | ... | Dunedin, Otago | 136.0 | My daughter is 25 and has recently been diagno... | Flights to Mexico and operation for Stemcell R... | Created by, and paying to a verified bank acco... | Page Moderated | NaN | NaN | NaN | Em’s MS Journey Stemcell replacement for Em’s ... |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 2016-05-02 | Amber and Daniel were twins born on 24th and 2... | Raising money for the little McSweeney family ... | https://static.givealittle.co.nz/assets/hero/a... | https://givealittle.co.nz/cause/hartleytwinsmi... | 4834.00 | 0.0 | closed | Read more | https://givealittle.co.nz/cause/hartleytwinsmi... | 2 May 2016 | ... | Canterbury | 122.0 | PLEASE READ THE UPDATE TO THIS POST UNDER UPDA... | NaN | Created by, and paying to a verified bank acco... | Page Moderated | 1.0 | Attention all followers of our story\n31 May 2... | NaN | Amber and Daniel were twins born on 24th and 2... |
| 2016-05-01 | Tania Whanau, known by most as Tarnz, is a str... | Since February '16 Tarnz has undergone two ope... | https://static.givealittle.co.nz/assets/hero/6... | https://givealittle.co.nz/cause/fightbreastcan... | 1550.00 | 0.0 | closed | Read more | https://givealittle.co.nz/cause/fightbreastcan... | 1 May 2016 | ... | Hawke's Bay | 34.0 | Tania Whanau, known by most as Tarnz, is a str... | NaN | Created by_x000D_\n _x000D_\n Nanc... | Page Moderated | NaN | NaN | NaN | Tania Whanau, known by most as Tarnz, is a str... |
| 2016-05-01 | Cardiology Fundraiser | Fundraising for patients & families affected b... | https://static.givealittle.co.nz/assets/hero/b... | https://givealittle.co.nz/cause/fund4aheart | 915.00 | 0.0 | closed | Read more | https://givealittle.co.nz/cause/fund4aheart | 1 May 2016 | ... | Waikato | 9.0 | We aim to provide clients with the most up-to-... | NaN | Created by, and paying to a verified bank acco... | Page Moderated | NaN | NaN | NaN | Cardiology Fundraiser Fundraising for patients... |
| 2016-04-30 | Help for Julia, while she is caring for Ian Ro... | Please support me so I'll be able to continue ... | https://static.givealittle.co.nz/assets/hero/5... | https://givealittle.co.nz/cause/help4julia | 2645.00 | 0.0 | closed | Read more | https://givealittle.co.nz/cause/help4julia | 30 Apr 2016 | ... | Canterbury | 31.0 | I'm caring for my dear Kiwi friend Ian Roberts... | NaN | Created by, and paying to a verified bank acco... | Page Moderated | NaN | NaN | NaN | Help for Julia, while she is caring for Ian Ro... |
| 2016-04-29 | TRUS Biopsy Machine for Taranaki | The men of Taranaki need this TRUS Biopsy Mach... | https://static.givealittle.co.nz/assets/hero/c... | https://givealittle.co.nz/cause/prostatebiopsy... | 1910.00 | 20000.0 | closed | Read more | https://givealittle.co.nz/cause/prostatebiopsy... | 29 Apr 2016 | ... | Taranaki | 15.0 | Currently the TRUS (Trans Rectal Ultrasound) m... | NaN | Created by, and paying to a verified bank acco... | Page Moderated | NaN | NaN | NaN | TRUS Biopsy Machine for Taranaki The men of Ta... |
10418 rows × 22 columns
In [2]:
df.amountRaised.resample("1YE").sum().plot.bar()
✔️ 267 ms (2024-11-04T13:24:37/2024-11-04T13:24:37)
In [3]:
keywords = ["covid", "cancer", "leukemia", "heart", "stroke", "kidney"]
for word in keywords:
df[word] = df.text.str.lower().str.count(rf"\b{word}\b")
df
✔️ 2.21 s (2024-11-04T13:24:38/2024-11-04T13:24:40)
Out[3]:
| title | pitch | hero | uri | amountRaised | goal | timeLeft | action | actionUri | start | ... | n_updates | updates | Q&A | text | covid | cancer | leukemia | heart | stroke | kidney | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| start | |||||||||||||||||||||
| 2024-11-03 | Young mother of 3 recently diagnosed with aggr... | Young mother of three fighting aggresive breas... | https://static.givealittle.co.nz/assets/hero/e... | https://givealittle.co.nz/cause/young-mother-o... | 3900.00 | 0.0 | 181 days to go | Donate | https://givealittle.co.nz/donate/cause/young-m... | 3 Nov 2024 | ... | NaN | NaN | NaN | Young mother of 3 recently diagnosed with aggr... | 0 | 3 | 0 | 0 | 0 | 0 |
| 2024-11-01 | Finlee Bremners Road to recovery | Helping our great friends navigate Finlee's lo... | https://static.givealittle.co.nz/assets/hero/2... | https://givealittle.co.nz/cause/finlee-bremner... | 4500.00 | 0.0 | 177 days to go | Donate | https://givealittle.co.nz/donate/cause/finlee-... | 1 Nov 2024 | ... | NaN | NaN | NaN | Finlee Bremners Road to recovery Helping our g... | 0 | 0 | 0 | 0 | 0 | 1 |
| 2024-10-31 | Paula Matafahi's Grade 4 Brain Cancer Fight - ... | This page has been set up to help Paula access... | https://static.givealittle.co.nz/assets/hero/b... | https://givealittle.co.nz/cause/paulas-stage-4... | 47522.15 | 55000.0 | 27 days to go | Donate | https://givealittle.co.nz/donate/cause/paulas-... | 31 Oct 2024 | ... | 2.0 | Nearly there!\n3 November 2024\nWe are just ab... | NaN | Paula Matafahi's Grade 4 Brain Cancer Fight - ... | 0 | 3 | 0 | 0 | 0 | 0 |
| 2024-10-31 | Our Sweet boy Jairo | Please help Jairo get the testing and screenin... | https://static.givealittle.co.nz/assets/hero/2... | https://givealittle.co.nz/cause/our-sweet-boy-... | 925.00 | 1000.0 | 19 days to go | Donate | https://givealittle.co.nz/donate/cause/our-swe... | 31 Oct 2024 | ... | 1.0 | Follow up appointments\n2 November 2024\nJust ... | NaN | Our Sweet boy Jairo Please help Jairo get the ... | 0 | 0 | 0 | 0 | 0 | 0 |
| 2024-10-30 | Em’s MS Journey | Stemcell replacement for Em’s MS Journey. Clín... | https://static.givealittle.co.nz/assets/hero/6... | https://givealittle.co.nz/cause/ems-ms-journey | 9640.00 | 90000.0 | 178 days to go | Donate | https://givealittle.co.nz/donate/cause/ems-ms-... | 30 Oct 2024 | ... | NaN | NaN | NaN | Em’s MS Journey Stemcell replacement for Em’s ... | 0 | 0 | 0 | 0 | 0 | 0 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 2016-05-02 | Amber and Daniel were twins born on 24th and 2... | Raising money for the little McSweeney family ... | https://static.givealittle.co.nz/assets/hero/a... | https://givealittle.co.nz/cause/hartleytwinsmi... | 4834.00 | 0.0 | closed | Read more | https://givealittle.co.nz/cause/hartleytwinsmi... | 2 May 2016 | ... | 1.0 | Attention all followers of our story\n31 May 2... | NaN | Amber and Daniel were twins born on 24th and 2... | 0 | 0 | 0 | 0 | 0 | 0 |
| 2016-05-01 | Tania Whanau, known by most as Tarnz, is a str... | Since February '16 Tarnz has undergone two ope... | https://static.givealittle.co.nz/assets/hero/6... | https://givealittle.co.nz/cause/fightbreastcan... | 1550.00 | 0.0 | closed | Read more | https://givealittle.co.nz/cause/fightbreastcan... | 1 May 2016 | ... | NaN | NaN | NaN | Tania Whanau, known by most as Tarnz, is a str... | 0 | 4 | 0 | 0 | 0 | 0 |
| 2016-05-01 | Cardiology Fundraiser | Fundraising for patients & families affected b... | https://static.givealittle.co.nz/assets/hero/b... | https://givealittle.co.nz/cause/fund4aheart | 915.00 | 0.0 | closed | Read more | https://givealittle.co.nz/cause/fund4aheart | 1 May 2016 | ... | NaN | NaN | NaN | Cardiology Fundraiser Fundraising for patients... | 0 | 0 | 0 | 5 | 0 | 0 |
| 2016-04-30 | Help for Julia, while she is caring for Ian Ro... | Please support me so I'll be able to continue ... | https://static.givealittle.co.nz/assets/hero/5... | https://givealittle.co.nz/cause/help4julia | 2645.00 | 0.0 | closed | Read more | https://givealittle.co.nz/cause/help4julia | 30 Apr 2016 | ... | NaN | NaN | NaN | Help for Julia, while she is caring for Ian Ro... | 0 | 4 | 0 | 1 | 0 | 0 |
| 2016-04-29 | TRUS Biopsy Machine for Taranaki | The men of Taranaki need this TRUS Biopsy Mach... | https://static.givealittle.co.nz/assets/hero/c... | https://givealittle.co.nz/cause/prostatebiopsy... | 1910.00 | 20000.0 | closed | Read more | https://givealittle.co.nz/cause/prostatebiopsy... | 29 Apr 2016 | ... | NaN | NaN | NaN | TRUS Biopsy Machine for Taranaki The men of Ta... | 0 | 6 | 0 | 0 | 0 | 0 |
10418 rows × 28 columns
In [4]:
df[keywords].describe()
✔️ 26.6 ms (2024-11-04T13:24:40/2024-11-04T13:24:40)
Out[4]:
| covid | cancer | leukemia | heart | stroke | kidney | |
|---|---|---|---|---|---|---|
| count | 10418.000000 | 10418.000000 | 10418.000000 | 10418.000000 | 10418.000000 | 10418.000000 |
| mean | 0.068823 | 1.349203 | 0.041083 | 0.321367 | 0.072663 | 0.111058 |
| std | 0.427711 | 2.349014 | 0.340030 | 1.050700 | 0.532337 | 0.782261 |
| min | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 |
| 25% | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 |
| 50% | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 |
| 75% | 0.000000 | 2.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 |
| max | 9.000000 | 25.000000 | 8.000000 | 15.000000 | 23.000000 | 23.000000 |
In [5]:
df.resample("1M")[keywords].sum().plot(height=800)
✔️ 83.2 ms (2024-11-04T13:24:40/2024-11-04T13:24:40)
In [6]:
topic_model = BERTopic(representation_model=KeyBERTInspired(), min_topic_size=10, verbose=True)
topic_model.fit(df.text)
topic_model.get_topic_info()
✔️ 44.2 s (2024-11-04T13:24:40/2024-11-04T13:25:24)
2024-11-04 13:24:40,756 - BERTopic - Embedding - Transforming documents to embeddings.
Batches: 0%| | 0/326 [00:00<?, ?it/s]
2024-11-04 13:24:56,858 - BERTopic - Embedding - Completed ✓ 2024-11-04 13:24:56,859 - BERTopic - Dimensionality - Fitting the dimensionality reduction algorithm 2024-11-04 13:25:18,314 - BERTopic - Dimensionality - Completed ✓ 2024-11-04 13:25:18,323 - BERTopic - Cluster - Start clustering the reduced embeddings 2024-11-04 13:25:18,727 - BERTopic - Cluster - Completed ✓ 2024-11-04 13:25:18,735 - BERTopic - Representation - Extracting topics from clusters using representation models. 2024-11-04 13:25:23,828 - BERTopic - Representation - Completed ✓
Out[6]:
| Topic | Count | Name | Representation | Representative_Docs | |
|---|---|---|---|---|---|
| 0 | -1 | 4205 | -1_cancer_treatment_surgery_medical | [cancer, treatment, surgery, medical, support,... | [To help a grieving widow and her children to ... |
| 1 | 0 | 2251 | 0_cancer_chemotherapy_diagnosed_chemo | [cancer, chemotherapy, diagnosed, chemo, treat... | [Living life by minutes A mother fighting agai... |
| 2 | 1 | 302 | 1_recovery_rehabilitation_rehab_stroke | [recovery, rehabilitation, rehab, stroke, inju... | [Support for an Awesome family in a time of ne... |
| 3 | 2 | 266 | 2_coronary_medical_insurance_hospital | [coronary, medical, insurance, hospital, fathe... | [To raise the fund for my father, due to the u... |
| 4 | 3 | 252 | 3_family_accommodation_expenses_support | [family, accommodation, expenses, support, wha... | [Pete Waipouri & Whanau - Help us bring Pete h... |
| ... | ... | ... | ... | ... | ... |
| 72 | 71 | 11 | 71_barre_paralysis_syndrome_disease | [barre, paralysis, syndrome, disease, rehabili... | [Help Alex Reilly On His Road To Recovery Help... |
| 73 | 72 | 11 | 72_tonsillectomy_surgery_adenoidectomy_otolary... | [tonsillectomy, surgery, adenoidectomy, otolar... | [Arachnoid cyst surgery for my child Please he... |
| 74 | 73 | 11 | 73_boxing_charitable_auckland_fundraiser | [boxing, charitable, auckland, fundraiser, spo... | [Wellington IT Heavy Hitters 2019 Suicide Prev... |
| 75 | 74 | 11 | 74_funds_expenses_finances_zoe | [funds, expenses, finances, zoe, gail, aid, he... | [Give for Gail Helping Gail get back on her fe... |
| 76 | 75 | 10 | 75_autism_disorders_syndrome_disorder | [autism, disorders, syndrome, disorder, neurod... | [Fundraiser for Autism Let's Make Autism Accep... |
77 rows × 5 columns
In [7]:
topic_model.visualize_topics()
In [8]:
topics_over_time = topic_model.topics_over_time(df.text, df.index, nr_bins=10)
topics_over_time
✔️ 27.3 s (2024-11-04T13:25:29/2024-11-04T13:25:56)
10it [00:27, 2.72s/it]
Out[8]:
| Topic | Words | Frequency | Timestamp | |
|---|---|---|---|---|
| 0 | -1 | treatment, cancer, medical, surgery, hospital | 389 | 2016-04-25 21:21:36 |
| 1 | 0 | cancer, chemotherapy, diagnosed, treatments, c... | 165 | 2016-04-25 21:21:36 |
| 2 | 1 | recovery, rehabilitation, support, injury, funds | 33 | 2016-04-25 21:21:36 |
| 3 | 2 | hospital, expenses, assistance, nz, medical | 13 | 2016-04-25 21:21:36 |
| 4 | 3 | whanau, family, maori, support, auckland | 18 | 2016-04-25 21:21:36 |
| ... | ... | ... | ... | ... |
| 663 | 65 | adhd, diagnosis, autism, assessment, diagnostic | 3 | 2023-12-28 00:00:00 |
| 664 | 68 | cancer, battle, aunt, mom, struggle | 1 | 2023-12-28 00:00:00 |
| 665 | 71 | parkinsons, hamish, barre, disease, syndrome | 1 | 2023-12-28 00:00:00 |
| 666 | 72 | tonsillectomy, nixies, nixie, tonsils, tonsil | 1 | 2023-12-28 00:00:00 |
| 667 | 75 | neurodiversity, neurodiverse, neurodivergent, ... | 1 | 2023-12-28 00:00:00 |
668 rows × 4 columns
In [9]:
topic_model.visualize_topics_over_time(topics_over_time, top_n_topics=20, height=600)
✔️ 73.6 ms (2024-11-04T13:25:56/2024-11-04T13:25:56)